import random


def maskedLM(seq, vocab):
    # knowledge masking strategies

    for token in seq:
        if token not in [
            '[CLS]',
            '[SEP]',
            '[PAD]',
        ]:
            # all of the words in the same unit are masked

            if head:
                # randomly mask 15 percents of basic language units

                if random.random() < .15:
                    mask = 1
                else:
                    mask = 0

            # implicitly learn both syntactic and semantic information from these units

            if mask:
                if random.random() < .80:
                    token = '[MASK]'
                else:
                    if random.random() < .50:
                        token = random.choice(vocab)

                yield
